#!/bin/tcsh -ef
####
#
#############################################################################
#                                                                           #
# Title: Synchronize Project with Backup                                    #
#                                                                           #
# (C) 2dx.org, GNU Plublic License.                                         #
#                                                                           #
# Created..........: 30/08/2007                                             #
# Last Modification: 22/09/2011                                             #
# Author...........: 2dx.org                                                #
#                                                                           #
#############################################################################
#
# SORTORDER: 35
#
# MANUAL: This script allows to copy the significant files from the currently selected images in this project into a different location, or vice versa. You can use this to synchronize your current working copy of this project with that on a remote file server. This could, for example, allow you to work on one common project with several people. For this, you would in the morning synchronize your copy with the remote server, by issuing an "UPDATE" command. This would pull the newer versions of images and image processing data from the server to your local copy. In the evening, when you are done, you could issue a "COMMIT" command, to push your changes to the backup location. 
#
# MANUAL: This script thereby compares the creation date of the local 2dx_image.cfg files with those on the backup location, and pushes or pulls an entire image file structure over only if the target 2dx_image.cfg file is older than the source file.
#
# MANUAL: You could also use this script to create a backup for your project, for file safety.
#
# DISPLAY: backup_also_APH
# DISPLAY: backup_also_Movie
# DISPLAY: backup_dir
# DISPLAY: backup_merge
# DISPLAY: backup_sure
# DISPLAY: backup_sync_deletions
# DISPLAY: backup_all_files
# DISPLAY: backup_large_images
# DISPLAY: backup_direction
#
#$end_local_vars
#
set bin_2dx = ""
set proc_2dx = ""
#
#
set backup_also_APH = ""
set backup_also_Movie = ""
set backup_dir = ""
set backup_merge = ""
set backup_sure = ""
set backup_direction = ""
set backup_all_files = ""
set backup_large_images = ""
set backup_sync_deletions = ""
#
#$end_vars
#
set scriptname = 2dx_backup
\rm -f LOGS/${scriptname}.results
#
set ccp4_setup = 'y'
source ${proc_2dx}/initialize
source ${proc_2dx}/2dx_merge_makedirs 
#
echo backup_dir = ${backup_dir}
echo backup_direction = ${backup_direction}
echo backup_merge = ${backup_merge}
echo backup_all_files = ${backup_all_files}
echo backup_sure = ${backup_sure}
#
echo "<<@progress: 1>>"
#
echo "set backup_sure = no" >> LOGS/${scriptname}.results
#
set copy_command = "cp -f"
#
if ( ! -d ${backup_dir} ) then
  #
  ${proc_2dx}/linblock "The directory ${backup_dir} does not exist"
  set backup_root = `dirname ${backup_dir}`
  echo "backup root is ${backup_root}" 
  if (  -d ${backup_root} && ( ${backup_direction} == 0 || ${backup_direction} == 1 )) then
    ${proc_2dx}/linblock "Creating backup directory to submit the data to: ${backup_dir}"
    mkdir ${backup_dir}
  else
    ${proc_2dx}/linblock "Provide an existing directory path"
    ${proc_2dx}/protest "${backup_dir} not found"
  endif
  #
endif
#
if ( ${backup_sure} == "no" ) then
  ${proc_2dx}/protest "Not running (Are you sure is 'no')."
endif
#
set oldir = ${PWD}
#
cd ..
set local_root_dir = ${PWD}
set local_root_dir_noslash = `echo ${local_root_dir} | sed "s/\//#/g"`
echo local_root_dir = ${local_root_dir}
echo local_root_dir_noslash = ${local_root_dir_noslash}
set backup_dir_noslash = `echo ${backup_dir} | sed "s/\//#/g"`
echo backup_dir = ${backup_dir}
echo backup_dir_noslash = ${backup_dir_noslash}
cd ${oldir}
#
if ( ${backup_dir_noslash} == ${local_root_dir_noslash} ) then
  ${proc_2dx}/protest "ERROR: You cannot sync with the local directory."
endif
#
if ( ${backup_direction} == "0" || ${backup_direction} == 1 ) then
  ${proc_2dx}/linblock "Pushing local files to remote location."
else
  ${proc_2dx}/linblock "Pulling remote files to local working copy."
endif
#
if ( ${backup_merge} == "yes" ) then
  ############################################################################# 
  ${proc_2dx}/linblock "Copying merge directory"
  #############################################################################
  #
  if ( ${backup_direction} == "0" || ${backup_direction} == 1 ) then
    set from_dir = ${PWD}
    set to_dir = `echo ${from_dir} | sed "s/\//#/g" | sed "s/${local_root_dir_noslash}/${backup_dir_noslash}/" | sed "s/#/\//g"`
    if ( ! -d ${to_dir} ) then
      \mkdir -p ${to_dir}
      if ( ! -d ${to_dir} ) then
        ${proc_2dx}/protest "ERROR: ${to_dir} could not be created."
      endif
    endif
  else
    set to_dir = ${PWD}
    set from_dir = `echo ${to_dir} | sed "s/\//#/g" | sed "s/${local_root_dir_noslash}/${backup_dir_noslash}/" | sed "s/#/\//g"`
    if ( ! -d ${to_dir} ) then
      ${proc_2dx}/protest "ERROR: ${from_dir} not found."
    endif
  endif
  #
  echo "dummy" > ${from_dir}/SELECT-dummy.dat
  ${copy_command} ${from_dir}/SELECT* ${to_dir}
  \rm -f ${from_dir}/SELECT-dummy.dat
  \rm -f ${to_dir}/SELECT-dummy.dat
  #
  echo "dummy" > ${from_dir}/volume-dummy.tmp
  ${copy_command} ${from_dir}/volume* ${to_dir}
  \rm -f ${from_dir}/volume-dummy.tmp
  \rm -f ${to_dir}/volume-dummy.tmp
  #
  echo "dummy" > ${from_dir}/2dx_merge_dummy.tmp
  ${copy_command} ${from_dir}/2dx_merge* ${to_dir}
  \rm -f ${from_dir}/2dx_merge_dummy.tmp
  \rm -f ${to_dir}/2dx_merge_dummy.tmp
  #
  echo ":New: ${to_dir}/2dx_merge.cfg"
  ${copy_command} ${from_dir}/2dx_merge.cfg ${to_dir}
  #
  if ( ${backup_all_files} == "0" ) then
    if ( ${backup_direction} == "0" || ${backup_direction} == 1 ) then
      cat ${from_dir}/2dx_merge_dirfile.dat | sed "s/\//#/g" | sed "s/${local_root_dir_noslash}/${backup_dir_noslash}/" | sed "s/#/\//g" > ${to_dir}/2dx_merge_dirfile.dat
    else
      cat ${from_dir}/2dx_merge_dirfile.dat | sed "s/\//#/g" | sed "s/${backup_dir_noslash}/${local_root_dir_noslash}/" | sed "s/#/\//g" > ${to_dir}/2dx_merge_dirfile.dat
    endif
  else
    if ( ${backup_direction} == "0" || ${backup_direction} == 1 ) then
      cat ${from_dir}/2dx_merge_dirfile.dat | sed "s/\//#/g" | sed "s/${local_root_dir_noslash}/${backup_dir_noslash}/" | sed "s/#/\//g" > ${to_dir}/2dx_merge_dirfile.dat
    else
      if ( ! -e ${to_dir}/2dx_merge_dirfile.dat ) then
        cat ${from_dir}/2dx_merge_dirfile.dat | sed "s/\//#/g" | sed "s/${backup_dir_noslash}/${local_root_dir_noslash}/" | sed "s/#/\//g" > ${to_dir}/2dx_merge_dirfile.dat
      endif
    endif
  endif
  #
  if ( -d ${from_dir}/REGISTERS ) then
    echo ":New: ${to_dir}/REGISTERS"
    ${copy_command} -r ${from_dir}/REGISTERS ${to_dir}
  endif
  #
  source ${proc_2dx}/2dx_backup_sub.com ${from_dir} ${to_dir} y merge2D_MRClefthanded.mtz
  source ${proc_2dx}/2dx_backup_sub.com ${from_dir} ${to_dir} y merge2Dref_MRClefthanded.mtz
  source ${proc_2dx}/2dx_backup_sub.com ${from_dir} ${to_dir} y merge3D_MRClefthanded.mtz
  source ${proc_2dx}/2dx_backup_sub.com ${from_dir} ${to_dir} y merge3Dref_MRClefthanded.mtz
  source ${proc_2dx}/2dx_backup_sub.com ${from_dir} ${to_dir} y merge.aph
  source ${proc_2dx}/2dx_backup_sub.com ${from_dir} ${to_dir} y History.dat
  #
  if ( -d ${from_dir}/APH ) then
    if ( ! -d ${to_dir}/APH ) then
      \mkdir ${to_dir}/APH
    endif
    echo dummy > ${from_dir}/APH/dummy.tmp
    echo ":New: ${to_dir}/APH/*"
    ${copy_command} ${from_dir}/APH/* ${to_dir}/APH
    \rm ${from_dir}/APH/dummy.tmp
    \rm ${to_dir}/APH/dummy.tmp
  endif
  #
  if ( -e ${from_dir}/2dx_merge_dirfile.dat ) then
    echo ":New: ${to_dir}/2dx_merge_dirfile.dat"
    ${copy_command} ${from_dir}/2dx_merge_dirfil*.dat ${to_dir}
  endif
  #
endif
#
############################################################################# 
${proc_2dx}/linblock "Copying all directories"
#############################################################################
#
if ( ${backup_direction} == "0" || ${backup_direction} == 1 ) then
  cd ${oldir}/..
  set from_dir = ${PWD}
  set to_dir = `echo ${from_dir} | sed "s/\//#/g" | sed "s/${local_root_dir_noslash}/${backup_dir_noslash}/" | sed "s/#/\//g"`
  if ( ! -d ${to_dir} ) then
    \mkdir -p ${to_dir}
    if ( ! -d ${to_dir} ) then
      ${proc_2dx}/protest "ERROR: ${to_dir} could not be created."
    endif
  endif
  cd ${oldir}
else
  cd ..
  set to_dir = ${PWD}
  set from_dir = `echo ${to_dir} | sed "s/\//#/g" | sed "s/${local_root_dir_noslash}/${backup_dir_noslash}/" | sed "s/#/\//g"`
  if ( ! -d ${to_dir} ) then
    ${proc_2dx}/protest "ERROR: ${from_dir} not found."
  endif
  cd ${oldir}
endif
#
echo ":: "
echo "::Copying files from ${from_dir}"
echo "::to ${to_dir}."
echo ":: "
#
if ( ${backup_direction} == "0" || ${backup_direction} == 1 ) then
  # Push files
  if ( ${backup_all_files} == "0" ) then
    cd ..
    find . -name 2dx_image.cfg -print | sed 's/\/2dx_image.cfg//' > ${oldir}/SCRATCH/2dx_merge_alldirs.dat
    cd ${oldir}
  else
    ${copy_command} 2dx_merge_dirfile.dat ${oldir}/SCRATCH/2dx_merge_alldirs.dat
  endif
else
  # Pull files
  if ( ${backup_all_files} == "0" ) then
    # Pull all with 2dx_image.cfg
    cd ${backup_dir}
    find . -name 2dx_image.cfg -print | sed 's/\/2dx_image.cfg//' > ${oldir}/SCRATCH/2dx_merge_alldirs.dat
    cd ${oldir}
  else
    # Pull only those that are in the LOCAL 2dx_merge_dirfile.dat.
    cat ${oldir}/2dx_merge_dirfile.dat | sed "s/\//#/g" | sed "s/${local_root_dir_noslash}/${backup_dir_noslash}/" | sed "s/#/\//g" > ${oldir}/SCRATCH/2dx_merge_alldirs.dat
  endif
endif
#
@ dirnum = `cat SCRATCH/2dx_merge_alldirs.dat | wc -l`
echo ": Will work on ${dirnum} image directories."
set increment = `echo ${dirnum} | awk '{ s = 80.0 / ( $1 + 1 ) } END { print s }'`
set curval = 10.0
#
if ( ${dirnum} == "0" ) then
  ${proc_2dx}/protest "Copying not possible, since no 2dx_image.cfg exists."
endif
#
@ dircounter = 1
while ( ${dircounter} <= ${dirnum} )
  set from_dirfile = `sed -n "${dircounter}p" ${oldir}/SCRATCH/2dx_merge_alldirs.dat`
  cd ${from_dir}
  if ( -d ${from_dirfile} ) then
    cd ${from_dirfile}
    set from_dirfile = ${PWD}
    echo ":: "
    echo ":##############################################################################"
    echo "::Working on source directory: ${from_dirfile}"
    echo ":##############################################################################"
    #
    if ( ${backup_direction} == "0" || ${backup_direction} == 1 ) then
      set to_dirfile = `echo ${from_dirfile} | sed "s/\//#/g" | sed "s/${local_root_dir_noslash}/${backup_dir_noslash}/" | sed "s/#/\//g"`
    else
      set to_dirfile = `echo ${from_dirfile} | sed "s/\//#/g" | sed "s/${backup_dir_noslash}/${local_root_dir_noslash}/" | sed "s/#/\//g"`
    endif
    echo ":Copy ${from_dirfile}"
    echo ": to  ${to_dirfile}"
    #    
    set curval = `echo ${curval} ${increment} | awk '{ s = $1 + $2 } END { print s }'`
    set curprogress = `echo ${curval} | awk '{ s = int( $1 ) } END { print s }'`
    echo "<<@progress:${curprogress}>>"
    #
    cd ${from_dirfile}
    set docopy = 0
    if ( -e 2dx_image.cfg ) then
      if ( ${backup_direction} == "1" || ${backup_direction} == 3 ) then
        echo "::Copying this image directory."
        set docopy = 1
      else
        if ( -e ${to_dirfile}/2dx_image.cfg ) then
          set newer_file = `ls -t -1 2dx_image.cfg ${to_dirfile}/2dx_image.cfg | head -n 1`
          if ( ${newer_file} == "2dx_image.cfg" ) then
            echo "::Copying this image directory, since 2dx_image.cfg is newer than at the destination."
            set docopy = 1
          else
            echo "::NOT copying this image directory, since 2dx_image.cfg is older than at the destination."
            set docopy = 0
          endif
        else
          echo "::Copying this image directory, since it doesn't exist at destination."
          set docopy = 1
        endif
      endif
      if ( ${docopy} == "1" ) then
        #
        set loc_imagename = `fgrep "set imagename = " 2dx_image.cfg | cut -d\" -f2`
        set loc_nonmaskimagename = `fgrep "set nonmaskimagename = " 2dx_image.cfg | cut -d\" -f2`
        #
        if ( ! -d ${to_dirfile} ) then
          \mkdir -p ${to_dirfile}
          if ( ! -d ${to_dirfile} ) then
            ${proc_2dx}/protest "ERROR: ${to_dirfile} could not be created."
          endif
        endif
        #
        if ( ${backup_large_images} == "0" || ${backup_large_images} == "2" ) then
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} y 2dx_image.cfg
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} y 2dx_image.cfg-backup1
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} y 2dx_image.cfg-backup2
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} y 2dx_image.cfg-backup3
        endif
        #
        if ( ${backup_large_images} == "0" || ${backup_large_images} == "1" ) then
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} n ${loc_nonmaskimagename}.tif
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} n ${loc_nonmaskimagename}_raw.mrc
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} n ${loc_nonmaskimagename}.mrc
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} n ${loc_nonmaskimagename}_stack.mrc
        endif
        #
        if ( ${backup_large_images} == "0" || ${backup_large_images} == "2" ) then
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} y ${loc_nonmaskimagename}.spt
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} y ${loc_nonmaskimagename}_automask.spt
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} y ${loc_nonmaskimagename}_manualmask.spt
        endif
        #
        if ( ${backup_large_images} == "0" || ${backup_large_images} == "2" ) then
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} y History.dat
          source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile} ${to_dirfile} y ${loc_nonmaskimagename}_profile.dat
          #
          if ( ${backup_also_APH} == "0" ) then
            if ( -d ${from_dirfile}/APH ) then
              if ( ! -d ${to_dirfile}/APH ) then
                \mkdir ${to_dirfile}/APH
              endif
              echo dummy > ${from_dirfile}/APH/dummy.tmp
              echo ":New: ${to_dirfile}/APH/*"
              cd ${from_dirfile}/APH ; tar -cf - . | (cd ${to_dirfile}/APH ; tar -xpf -)
              # ${copy_command} ${from_dirfile}/APH/* ${to_dirfile}/APH
              \rm ${from_dirfile}/APH/dummy.tmp
              \rm ${to_dirfile}/APH/dummy.tmp
            endif
          endif
        endif

        if ( ${backup_large_images} == "0" || ${backup_large_images} == "2" ) then
          if ( ${backup_also_Movie} == "0" ) then
            source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile}/MovieA ${to_dirfile}/MA y direct_sum_filt_upscale.mrc
            source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile}/MovieA ${to_dirfile}/MA y direct_sum_fft.mrc
            source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile}/MovieB ${to_dirfile}/MB y direct_sum_filt_upscale.mrc
            source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile}/MovieB ${to_dirfile}/MB y direct_sum_fft.mrc
            source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile}/MA ${to_dirfile}/MA y direct_sum_filt_upscale.mrc
            source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile}/MA ${to_dirfile}/MA y direct_sum_fft.mrc
            source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile}/MB ${to_dirfile}/MB y direct_sum_filt_upscale.mrc
            source ${proc_2dx}/2dx_backup_sub.com ${from_dirfile}/MB ${to_dirfile}/MB y direct_sum_fft.mrc
          endif
        endif
	#
        if ( -e ${to_dirfile}/m${loc_nonmaskimagename}-profile.dat.gz ) then
          echo ":Removing: ${to_dirfile}/m${loc_nonmaskimagename}-profile.dat.gz"
          \rm ${to_dirfile}/m${loc_nonmaskimagename}-profile.dat.gz
        endif
        if ( -e ${to_dirfile}/m${loc_nonmaskimagename}.mtz ) then
          echo ":Removing: ${to_dirfile}/m${loc_nonmaskimagename}.mtz"
          \rm ${to_dirfile}/m${loc_nonmaskimagename}.mtz
        endif
        if ( -e ${to_dirfile}/m${loc_nonmaskimagename}.spt ) then
          echo ":Removing: ${to_dirfile}/m${loc_nonmaskimagename}.spt"
          \rm ${to_dirfile}/m${loc_nonmaskimagename}.spt
        endif
        #
      endif
      #
    endif
    #
  endif
  @ dircounter++ 
  cd ${oldir}
  #
  #
end
#
  if ( ${backup_sync_deletions} == "yes" ) then
    ############################################################################# 
    ${proc_2dx}/linblock "#"
    ${proc_2dx}/linblock "Synchronizing deletion of image directories:"
    ${proc_2dx}/linblock "#"
    #############################################################################
    echo ":Copy ${from_dir}"
    echo ": to  ${to_dir}"
    cd ${to_dir}
    echo "::Gathering directory list ..."
    find . -name 2dx_image.cfg -print | sed 's/\/2dx_image.cfg//' | grep -v TRASH | grep -v merge | sort > ${to_dir}/to_dirfile_list.dat
    set dirnum = `cat ${to_dir}/to_dirfile_list.dat | wc -l`
    echo ": Will check for deletions for ${dirnum} image directories."
    set dircounter = ${dirnum}
    while ( ${dircounter} > 0 )
      #
      ${bin_2dx}/2dx_getline.exe << eot > TMP.tmp
${to_dir}/to_dirfile_list.dat
${dircounter}
eot
      #
      set dir_name = `cat TMP.tmp`
      \rm TMP.tmp
      #
      set to_dirfile = `echo ${to_dir}/${dir_name}`
      set from_dirfile = `echo ${from_dir}/${dir_name}`
      if ( ! -d ${from_dirfile} ) then
         echo "::Moving ${to_dirfile} to TRASH on target destination"
         cd ${to_dir}
         if ( ! -d TRASH ) then
           \mkdir TRASH
         endif
         if ( -d TRASH/${dir_name} ) then
           \rm -rf TRASH/${dir_name}
         endif
         \mv -f ${to_dirfile} TRASH
         cd ${to_dir}
      else
         # echo "Both exist: ${to_dirfile}  ${from_dirfile}"
      endif
      @ dircounter -= 1
      cd ${to_dir}
    end
    \rm ${to_dir}/to_dirfile_list.dat
    #
  endif

echo "<<@progress:100>>"
#
############################################################################# 
${proc_2dx}/linblock "Done."
#############################################################################
#
#
#
